# encoding: utf-8

require_relative '../test_helper'

class TestAttendeeTokenizer < AttendeeTestCase

  def setup
    @wiki = [
      'Test [[Link|internal link]] and [http://example.com external link].',
      'Try __MAGIC__ with [[Multiline',
      'link (because we can)]].',
      '[[Category:cat1]]',
      'Link to [[:Category:cat2]].',
      '== Heading ==',
      '{{Template}}',
      'Function with {{#func|param|{{{var}}}}} and <nowiki>{{{var}}}</nowiki>!',
      '{{Multi',
      ' | line=1',
      ' | [[link]]',
      ' | {{{var}}}',
      '',
      '}}'
    ]

    @html = [
      'test <a>test</a> test',
      '<b>test <a>test</a></b>',
      'test <a test="test"><b>test</b></a>, test',
      '<a>test</a><b test="test">test</b><a>test</a>'
    ]
  end

  def test_basic
    meet({}, [
      'Dies ist ein Test.'
    ], [
      tk('Dies|WORD|0|0'),
      tk('ist|WORD|1|5'),
      tk('ein|WORD|2|9'),
      tk('Test|WORD|3|13'),
      tk('.|PUNC|4|17')
    ])
  end

  def test_complex
    meet({}, [
      '1964 www.vorhauer.de bzw. nasenbär, ()'
    ], [
      tk('1964|NUMS|0|0'),
      tk('www.vorhauer.de|URLS|1|5'),
      tk('bzw|WORD|2|21'),
      tk('.|PUNC|3|24'),
      tk('nasenbär|WORD|4|26'),
      tk(',|PUNC|5|35'),
      tk('(|OTHR|6|37'),
      tk(')|OTHR|7|38')
    ])
  end

  def test_hyphen
    meet({}, [
      'Di-es i-s-t -ein- -Test - mit- Binde--strich-.'
    ], [
      tk('Di-es|WORD|0|0'),
      tk('i-s-t|WORD|1|6'),
      tk('-|OTHR|2|12'),
      tk('ein|WORD|3|13'),
      tk('-|OTHR|4|16'),
      tk('-|OTHR|5|18'),
      tk('Test|WORD|6|19'),
      tk('-|OTHR|7|24'),
      tk('mit|WORD|8|26'),
      tk('-|OTHR|9|29'),
      tk('Binde--strich|WORD|10|31'),
      tk('-|OTHR|11|44'),
      tk('.|PUNC|12|45')
    ])
  end

  def test_wiki1
    meet({}, @wiki, [
      tk('Test|WORD|0|0'),
      tk('[|OTHR|1|5'),
      tk('[|OTHR|2|6'),
      tk('Link|WORD|3|7'),
      tk('||OTHR|4|11'),
      tk('internal|WORD|5|12'),
      tk('link|WORD|6|21'),
      tk(']|OTHR|7|25'),
      tk(']|OTHR|8|26'),
      tk('and|WORD|9|28'),
      tk('[|OTHR|10|32'),
      tk('http://example.com|URLS|11|33'),
      tk('external|WORD|12|52'),
      tk('link|WORD|13|61'),
      tk(']|OTHR|14|65'),
      tk('.|PUNC|15|66'),
      tk('Try|WORD|16|67'),
      tk('_|OTHR|17|71'),
      tk('_|OTHR|18|72'),
      tk('MAGIC|WORD|19|73'),
      tk('_|OTHR|20|78'),
      tk('_|OTHR|21|79'),
      tk('with|WORD|22|81'),
      tk('[|OTHR|23|86'),
      tk('[|OTHR|24|87'),
      tk('Multiline|WORD|25|88'),
      tk('link|WORD|26|97'),
      tk('(|OTHR|27|102'),
      tk('because|WORD|28|103'),
      tk('we|WORD|29|111'),
      tk('can|WORD|30|114'),
      tk(')|OTHR|31|117'),
      tk(']|OTHR|32|118'),
      tk(']|OTHR|33|119'),
      tk('.|PUNC|34|120'),
      tk('[|OTHR|35|121'),
      tk('[|OTHR|36|122'),
      tk('Category|WORD|37|123'),
      tk(':|PUNC|38|131'),
      tk('cat1|WORD|39|132'),
      tk(']|OTHR|40|136'),
      tk(']|OTHR|41|137'),
      tk('Link|WORD|42|138'),
      tk('to|WORD|43|143'),
      tk('[|OTHR|44|146'),
      tk('[|OTHR|45|147'),
      tk(':|PUNC|46|148'),
      tk('Category|WORD|47|149'),
      tk(':|PUNC|48|157'),
      tk('cat2|WORD|49|158'),
      tk(']|OTHR|50|162'),
      tk(']|OTHR|51|163'),
      tk('.|PUNC|52|164'),
      tk('=|OTHR|53|165'),
      tk('=|OTHR|54|166'),
      tk('Heading|WORD|55|168'),
      tk('=|OTHR|56|176'),
      tk('=|OTHR|57|177'),
      tk('{|OTHR|58|178'),
      tk('{|OTHR|59|179'),
      tk('Template|WORD|60|180'),
      tk('}|OTHR|61|188'),
      tk('}|OTHR|62|189'),
      tk('Function|WORD|63|190'),
      tk('with|WORD|64|199'),
      tk('{|OTHR|65|204'),
      tk('{|OTHR|66|205'),
      tk('#|OTHR|67|206'),
      tk('func|WORD|68|207'),
      tk('||OTHR|69|211'),
      tk('param|WORD|70|212'),
      tk('||OTHR|71|217'),
      tk('{|OTHR|72|218'),
      tk('{|OTHR|73|219'),
      tk('{|OTHR|74|220'),
      tk('var|WORD|75|221'),
      tk('}|OTHR|76|224'),
      tk('}|OTHR|77|225'),
      tk('}|OTHR|78|226'),
      tk('}|OTHR|79|227'),
      tk('}|OTHR|80|228'),
      tk('and|WORD|81|230'),
      tk('<|OTHR|82|234'),
      tk('nowiki|WORD|83|235'),
      tk('>|OTHR|84|241'),
      tk('{|OTHR|85|242'),
      tk('{|OTHR|86|243'),
      tk('{|OTHR|87|244'),
      tk('var|WORD|88|245'),
      tk('}|OTHR|89|248'),
      tk('}|OTHR|90|249'),
      tk('}|OTHR|91|250'),
      tk('<|OTHR|92|251'),
      tk('/|OTHR|93|252'),
      tk('nowiki|WORD|94|253'),
      tk('>|OTHR|95|259'),
      tk('!|PUNC|96|260'),
      tk('{|OTHR|97|261'),
      tk('{|OTHR|98|262'),
      tk('Multi|WORD|99|263'),
      tk('||OTHR|100|269'),
      tk('line|WORD|101|271'),
      tk('=|OTHR|102|275'),
      tk('1|NUMS|103|276'),
      tk('||OTHR|104|278'),
      tk('[|OTHR|105|280'),
      tk('[|OTHR|106|281'),
      tk('link|WORD|107|282'),
      tk(']|OTHR|108|286'),
      tk(']|OTHR|109|287'),
      tk('||OTHR|110|289'),
      tk('{|OTHR|111|291'),
      tk('{|OTHR|112|292'),
      tk('{|OTHR|113|293'),
      tk('var|WORD|114|294'),
      tk('}|OTHR|115|297'),
      tk('}|OTHR|116|298'),
      tk('}|OTHR|117|299'),
      tk('}|OTHR|118|300'),
      tk('}|OTHR|119|301')
    ])
  end

  def test_wiki2
    meet({ 'space' => true, 'tags' => true, 'wiki' => true }, @wiki, [
      tk('Test|WORD|0|0'),
      tk(' |SPAC|1|4'),
      tk('[[|WIKI|2|5'),
      tk('Link|internal link]]|WIKI|3|7'),
      tk(' |SPAC|4|27'),
      tk('and|WORD|5|28'),
      tk(' |SPAC|6|31'),
      tk('[http://|WIKI|7|32'),
      tk('example.com external link]|WIKI|8|40'),
      tk('.|PUNC|9|66'),
      tk('Try|WORD|10|67'),
      tk(' |SPAC|11|70'),
      tk('__MAGIC__|WIKI|12|71'),
      tk(' |SPAC|13|80'),
      tk('with|WORD|14|81'),
      tk(' |SPAC|15|85'),
      tk('[[|WIKI|16|86'),
      tk('Multiline|WIKI|17|88'),
      tk('link (because we can)]]|WIKI|18|97'),
      tk('.|PUNC|19|120'),
      tk('[[|WIKI|20|121'),
      tk('Category:cat1]]|WIKI|21|123'),
      tk('Link|WORD|22|138'),
      tk(' |SPAC|23|142'),
      tk('to|WORD|24|143'),
      tk(' |SPAC|25|145'),
      tk('[[|WIKI|26|146'),
      tk(':Category:cat2]]|WIKI|27|148'),
      tk('.|PUNC|28|164'),
      tk('== Heading ==|WIKI|29|165'),
      tk('{{|WIKI|30|178'),
      tk('Template}}|WIKI|31|180'),
      tk('Function|WORD|32|190'),
      tk(' |SPAC|33|198'),
      tk('with|WORD|34|199'),
      tk(' |SPAC|35|203'),
      tk('{{|WIKI|36|204'),
      tk('#func|param||WIKI|37|206'),
      tk('{{{|WIKI|38|218'),
      tk('var}}}|WIKI|39|221'),
      tk('}}|WIKI|40|227'),
      tk(' |SPAC|41|229'),
      tk('and|WORD|42|230'),
      tk(' |SPAC|43|233'),
      tk('<|HTML|44|234'),
      tk('nowiki>|HTML|45|235'),
      tk('{{{|WIKI|46|242'),
      tk('var}}}|WIKI|47|245'),
      tk('<|HTML|48|251'),
      tk('/nowiki>|HTML|49|252'),
      tk('!|PUNC|50|260'),
      tk('{{|WIKI|51|261'),
      tk('Multi|WIKI|52|263'),
      tk(' | line=1|WIKI|53|268'),
      tk(' | |WIKI|54|277'),
      tk('[[|WIKI|55|280'),
      tk('link]]|WIKI|56|282'),
      tk('|WIKI|57|288'),
      tk(' | |WIKI|58|288'),
      tk('{{{|WIKI|59|291'),
      tk('var}}}|WIKI|60|294'),
      tk('|WIKI|61|300'),
      tk('|WIKI|62|300'),
      tk('}}|WIKI|63|300')
    ])
  end

  def test_html1
    meet({ 'tags' => true }, @html, [
      tk('test|WORD|0|0'),
      tk('<|HTML|1|5'),
      tk('a>|HTML|2|6'),
      tk('test|WORD|3|8'),
      tk('<|HTML|4|12'),
      tk('/a>|HTML|5|13'),
      tk('test|WORD|6|17'),
      tk('<|HTML|7|21'),
      tk('b>|HTML|8|22'),
      tk('test|WORD|9|24'),
      tk('<|HTML|10|29'),
      tk('a>|HTML|11|30'),
      tk('test|WORD|12|32'),
      tk('<|HTML|13|36'),
      tk('/a>|HTML|14|37'),
      tk('<|HTML|15|40'),
      tk('/b>|HTML|16|41'),
      tk('test|WORD|17|44'),
      tk('<|HTML|18|49'),
      tk('a test="test">|HTML|19|50'),
      tk('<|HTML|20|64'),
      tk('b>|HTML|21|65'),
      tk('test|WORD|22|67'),
      tk('<|HTML|23|71'),
      tk('/b>|HTML|24|72'),
      tk('<|HTML|25|75'),
      tk('/a>|HTML|26|76'),
      tk(',|PUNC|27|79'),
      tk('test|WORD|28|81'),
      tk('<|HTML|29|85'),
      tk('a>|HTML|30|86'),
      tk('test|WORD|31|88'),
      tk('<|HTML|32|92'),
      tk('/a>|HTML|33|93'),
      tk('<|HTML|34|96'),
      tk('b test="test">|HTML|35|97'),
      tk('test|WORD|36|111'),
      tk('<|HTML|37|115'),
      tk('/b>|HTML|38|116'),
      tk('<|HTML|39|119'),
      tk('a>|HTML|40|120'),
      tk('test|WORD|41|122'),
      tk('<|HTML|42|126'),
      tk('/a>|HTML|43|127')
    ])
  end

  def test_html2
    meet({ 'skip-tags' => 'a' }, @html, [
      tk('test|WORD|0|0'),
      tk('<|SKIP|1|5'),
      tk('a>|SKIP|2|6'),
      tk('test|SKIP|3|8'),
      tk('<|SKIP|4|12'),
      tk('/a>|SKIP|5|13'),
      tk('test|WORD|6|17'),
      tk('<|HTML|7|21'),
      tk('b>|HTML|8|22'),
      tk('test|WORD|9|24'),
      tk('<|SKIP|10|29'),
      tk('a>|SKIP|11|30'),
      tk('test|SKIP|12|32'),
      tk('<|SKIP|13|36'),
      tk('/a>|SKIP|14|37'),
      tk('<|HTML|15|40'),
      tk('/b>|HTML|16|41'),
      tk('test|WORD|17|44'),
      tk('<|SKIP|18|49'),
      tk('a test="test">|SKIP|19|50'),
      tk('<|SKIP|20|64'),
      tk('b>|SKIP|21|65'),
      tk('test|SKIP|22|67'),
      tk('<|SKIP|23|71'),
      tk('/b>|SKIP|24|72'),
      tk('<|SKIP|25|75'),
      tk('/a>|SKIP|26|76'),
      tk(',|PUNC|27|79'),
      tk('test|WORD|28|81'),
      tk('<|SKIP|29|85'),
      tk('a>|SKIP|30|86'),
      tk('test|SKIP|31|88'),
      tk('<|SKIP|32|92'),
      tk('/a>|SKIP|33|93'),
      tk('<|HTML|34|96'),
      tk('b test="test">|HTML|35|97'),
      tk('test|WORD|36|111'),
      tk('<|HTML|37|115'),
      tk('/b>|HTML|38|116'),
      tk('<|SKIP|39|119'),
      tk('a>|SKIP|40|120'),
      tk('test|SKIP|41|122'),
      tk('<|SKIP|42|126'),
      tk('/a>|SKIP|43|127')
    ])
  end

end
